{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T14:34:54.278423Z",
     "start_time": "2020-04-03T14:34:54.266186Z"
    }
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):\n",
    "        super(TransformerModel, self).__init__()\n",
    "        from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
    "        self.model_type = 'Transformer'\n",
    "        self.src_mask = None\n",
    "        self.pos_encoder = PositionalEncoding(ninp, dropout)\n",
    "        encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)\n",
    "        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)\n",
    "        self.encoder = nn.Embedding(ntoken, ninp)\n",
    "        self.ninp = ninp\n",
    "        self.decoder = nn.Linear(ninp, ntoken)\n",
    "\n",
    "        self.init_weights()\n",
    "\n",
    "    def _generate_square_subsequent_mask(self, sz):\n",
    "        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)\n",
    "        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(\n",
    "            mask == 1, float(0.0))\n",
    "        return mask\n",
    "\n",
    "    def init_weights(self):\n",
    "        initrange = 0.1\n",
    "        self.encoder.weight.data.uniform_(-initrange, initrange)\n",
    "        self.decoder.bias.data.zero_()\n",
    "        self.decoder.weight.data.uniform_(-initrange, initrange)\n",
    "\n",
    "    def forward(self, src):\n",
    "        if self.src_mask is None or self.src_mask.size(0) != len(src):\n",
    "            device = src.device\n",
    "            mask = self._generate_square_subsequent_mask(len(src)).to(device)\n",
    "            self.src_mask = mask\n",
    "\n",
    "        src = self.encoder(src) * math.sqrt(self.ninp)\n",
    "        src = self.pos_encoder(src)\n",
    "        output = self.transformer_encoder(src, self.src_mask)\n",
    "        output = self.decoder(output)\n",
    "        return output\n",
    "\n",
    "\n",
    "class PositionalEncoding(nn.Module):\n",
    "    def __init__(self, d_model, dropout=0.1, max_len=5000):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(\n",
    "            torch.arange(0, d_model, 2).float() *\n",
    "            (-math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:x.size(0), :]\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T14:33:38.743476Z",
     "start_time": "2020-04-03T14:33:35.622886Z"
    }
   },
   "outputs": [],
   "source": [
    "import torchtext\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "TEXT = torchtext.data.Field(tokenize=get_tokenizer(\"basic_english\"),\n",
    "                            init_token='<sos>',\n",
    "                            eos_token='<eos>',\n",
    "                            lower=True)\n",
    "path = \"datasets/wikitext-2/\"\n",
    "train_txt, val_txt, test_txt = torchtext.datasets.WikiText2.splits(TEXT,\n",
    "                                                                   path=path)\n",
    "TEXT.build_vocab(train_txt)\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "\n",
    "def batchify(data, bsz):\n",
    "    data = TEXT.numericalize([data.examples[0].text])\n",
    "    nbatch = data.size(0) // bsz\n",
    "    data = data.narrow(0, 0, nbatch * bsz)\n",
    "    data = data.view(bsz, -1).t().contiguous()\n",
    "    return data.to(device)\n",
    "\n",
    "\n",
    "batch_size = 20\n",
    "eval_batch_size = 10\n",
    "train_data = batchify(train_txt, batch_size)\n",
    "val_data = batchify(val_txt, eval_batch_size)\n",
    "test_data = batchify(test_txt, eval_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T14:33:47.610199Z",
     "start_time": "2020-04-03T14:33:47.604843Z"
    }
   },
   "outputs": [],
   "source": [
    "bptt = 35\n",
    "\n",
    "\n",
    "def get_batch(source, i):\n",
    "    seq_len = min(bptt, len(source) - 1 - i)\n",
    "    data = source[i:i + seq_len]\n",
    "    target = source[i + 1:i + 1 + seq_len].view(-1)\n",
    "    return data, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T14:49:08.693135Z",
     "start_time": "2020-04-03T14:49:08.680348Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<eos>',\n",
       " '=',\n",
       " 'valkyria',\n",
       " 'chronicles',\n",
       " 'iii',\n",
       " '=',\n",
       " '<eos>',\n",
       " '<eos>',\n",
       " 'senjō',\n",
       " 'no',\n",
       " 'valkyria',\n",
       " '3',\n",
       " '<unk>',\n",
       " 'chronicles',\n",
       " '(',\n",
       " 'japanese',\n",
       " '戦場のヴァルキュリア3',\n",
       " ',',\n",
       " 'lit',\n",
       " '.',\n",
       " 'valkyria',\n",
       " 'of',\n",
       " 'the',\n",
       " 'battlefield',\n",
       " '3',\n",
       " ')',\n",
       " ',',\n",
       " 'commonly',\n",
       " 'referred',\n",
       " 'to',\n",
       " 'as',\n",
       " 'valkyria',\n",
       " 'chronicles',\n",
       " 'iii',\n",
       " 'outside',\n",
       " 'japan',\n",
       " ',',\n",
       " 'is',\n",
       " 'a',\n",
       " 'tactical',\n",
       " 'role',\n",
       " '@-@',\n",
       " 'playing',\n",
       " 'video',\n",
       " 'game',\n",
       " 'developed',\n",
       " 'by',\n",
       " 'sega',\n",
       " 'and',\n",
       " 'media',\n",
       " '.',\n",
       " 'vision',\n",
       " 'for',\n",
       " 'the',\n",
       " 'playstation',\n",
       " 'portable',\n",
       " '.',\n",
       " 'released',\n",
       " 'in',\n",
       " 'january',\n",
       " '2011',\n",
       " 'in',\n",
       " 'japan',\n",
       " ',',\n",
       " 'it',\n",
       " 'is',\n",
       " 'the',\n",
       " 'third',\n",
       " 'game',\n",
       " 'in',\n",
       " 'the',\n",
       " 'valkyria',\n",
       " 'series',\n",
       " '.',\n",
       " '<unk>',\n",
       " 'the',\n",
       " 'same',\n",
       " 'fusion',\n",
       " 'of',\n",
       " 'tactical',\n",
       " 'and',\n",
       " 'real',\n",
       " '@-@',\n",
       " 'time',\n",
       " 'gameplay',\n",
       " 'as',\n",
       " 'its',\n",
       " 'predecessors',\n",
       " ',',\n",
       " 'the',\n",
       " 'story',\n",
       " 'runs',\n",
       " 'parallel',\n",
       " 'to',\n",
       " 'the',\n",
       " 'first',\n",
       " 'game',\n",
       " 'and',\n",
       " 'follows',\n",
       " 'the',\n",
       " 'nameless',\n",
       " ',',\n",
       " 'a',\n",
       " 'penal',\n",
       " 'military',\n",
       " 'unit',\n",
       " 'serving',\n",
       " 'the',\n",
       " 'nation',\n",
       " 'of',\n",
       " 'gallia',\n",
       " 'during',\n",
       " 'the',\n",
       " 'second',\n",
       " 'europan',\n",
       " 'war',\n",
       " 'who',\n",
       " 'perform',\n",
       " 'secret',\n",
       " 'black',\n",
       " 'operations',\n",
       " 'and',\n",
       " 'are',\n",
       " 'pitted',\n",
       " 'against',\n",
       " 'the',\n",
       " 'imperial',\n",
       " 'unit',\n",
       " '<unk>',\n",
       " 'raven',\n",
       " '.',\n",
       " '<eos>',\n",
       " 'the',\n",
       " 'game',\n",
       " 'began',\n",
       " 'development',\n",
       " 'in',\n",
       " '2010',\n",
       " ',',\n",
       " 'carrying',\n",
       " 'over',\n",
       " 'a',\n",
       " 'large',\n",
       " 'portion',\n",
       " 'of',\n",
       " 'the',\n",
       " 'work',\n",
       " 'done',\n",
       " 'on',\n",
       " 'valkyria',\n",
       " 'chronicles',\n",
       " 'ii',\n",
       " '.',\n",
       " 'while',\n",
       " 'it',\n",
       " 'retained',\n",
       " 'the',\n",
       " 'standard',\n",
       " 'features',\n",
       " 'of',\n",
       " 'the',\n",
       " 'series',\n",
       " ',',\n",
       " 'it',\n",
       " 'also',\n",
       " 'underwent',\n",
       " 'multiple',\n",
       " 'adjustments',\n",
       " ',',\n",
       " 'such',\n",
       " 'as',\n",
       " 'making',\n",
       " 'the',\n",
       " 'game',\n",
       " 'more',\n",
       " '<unk>',\n",
       " 'for',\n",
       " 'series',\n",
       " 'newcomers',\n",
       " '.',\n",
       " 'character',\n",
       " 'designer',\n",
       " '<unk>',\n",
       " 'honjou',\n",
       " 'and',\n",
       " 'composer',\n",
       " 'hitoshi',\n",
       " 'sakimoto',\n",
       " 'both',\n",
       " 'returned',\n",
       " 'from',\n",
       " 'previous',\n",
       " 'entries',\n",
       " ',',\n",
       " 'along',\n",
       " 'with',\n",
       " 'valkyria',\n",
       " 'chronicles',\n",
       " 'ii',\n",
       " 'director',\n",
       " 'takeshi',\n",
       " 'ozawa',\n",
       " '.',\n",
       " 'a',\n",
       " 'large',\n",
       " 'team',\n",
       " 'of',\n",
       " 'writers',\n",
       " 'handled',\n",
       " 'the',\n",
       " 'script',\n",
       " '.',\n",
       " 'the',\n",
       " 'game',\n",
       " \"'\",\n",
       " 's',\n",
       " 'opening',\n",
       " 'theme',\n",
       " 'was',\n",
       " 'sung',\n",
       " 'by',\n",
       " 'may',\n",
       " \"'\",\n",
       " 'n',\n",
       " '.',\n",
       " '<eos>',\n",
       " 'it',\n",
       " 'met',\n",
       " 'with',\n",
       " 'positive',\n",
       " 'sales',\n",
       " 'in',\n",
       " 'japan',\n",
       " ',',\n",
       " 'and',\n",
       " 'was',\n",
       " 'praised',\n",
       " 'by',\n",
       " 'both',\n",
       " 'japanese',\n",
       " 'and',\n",
       " 'western',\n",
       " 'critics',\n",
       " '.',\n",
       " 'after',\n",
       " 'release',\n",
       " ',',\n",
       " 'it',\n",
       " 'received',\n",
       " 'downloadable',\n",
       " 'content',\n",
       " ',',\n",
       " 'along',\n",
       " 'with',\n",
       " 'an',\n",
       " 'expanded',\n",
       " 'edition',\n",
       " 'in',\n",
       " 'november',\n",
       " 'of',\n",
       " 'that',\n",
       " 'year',\n",
       " '.',\n",
       " 'it',\n",
       " 'was',\n",
       " 'also',\n",
       " 'adapted',\n",
       " 'into',\n",
       " 'manga',\n",
       " 'and',\n",
       " 'an',\n",
       " 'original',\n",
       " 'video',\n",
       " 'animation',\n",
       " 'series',\n",
       " '.',\n",
       " 'due',\n",
       " 'to',\n",
       " 'low',\n",
       " 'sales',\n",
       " 'of',\n",
       " 'valkyria',\n",
       " 'chronicles',\n",
       " 'ii',\n",
       " ',',\n",
       " 'valkyria',\n",
       " 'chronicles',\n",
       " 'iii',\n",
       " 'was',\n",
       " 'not',\n",
       " 'localized',\n",
       " ',',\n",
       " 'but',\n",
       " 'a',\n",
       " 'fan',\n",
       " 'translation',\n",
       " 'compatible',\n",
       " 'with',\n",
       " 'the',\n",
       " 'game',\n",
       " \"'\",\n",
       " 's',\n",
       " 'expanded',\n",
       " 'edition',\n",
       " 'was',\n",
       " 'released',\n",
       " 'in',\n",
       " '2014',\n",
       " '.',\n",
       " 'media',\n",
       " '.',\n",
       " 'vision',\n",
       " 'would',\n",
       " 'return',\n",
       " 'to',\n",
       " 'the',\n",
       " 'franchise',\n",
       " 'with',\n",
       " 'the',\n",
       " 'development',\n",
       " 'of',\n",
       " 'valkyria',\n",
       " 'azure',\n",
       " 'revolution',\n",
       " 'for',\n",
       " 'the',\n",
       " 'playstation',\n",
       " '4',\n",
       " '.',\n",
       " '<eos>',\n",
       " '<eos>',\n",
       " '=',\n",
       " '=',\n",
       " 'gameplay',\n",
       " '=',\n",
       " '=',\n",
       " '<eos>',\n",
       " '<eos>',\n",
       " 'as',\n",
       " 'with',\n",
       " 'previous',\n",
       " '<unk>',\n",
       " 'chronicles',\n",
       " 'games',\n",
       " ',',\n",
       " 'valkyria',\n",
       " 'chronicles',\n",
       " 'iii',\n",
       " 'is',\n",
       " 'a',\n",
       " 'tactical',\n",
       " 'role',\n",
       " '@-@',\n",
       " 'playing',\n",
       " 'game',\n",
       " 'where',\n",
       " 'players',\n",
       " 'take',\n",
       " 'control',\n",
       " 'of',\n",
       " 'a',\n",
       " 'military',\n",
       " 'unit',\n",
       " 'and',\n",
       " 'take',\n",
       " 'part',\n",
       " 'in',\n",
       " 'missions',\n",
       " 'against',\n",
       " 'enemy',\n",
       " 'forces',\n",
       " '.',\n",
       " 'stories',\n",
       " 'are',\n",
       " 'told',\n",
       " 'through',\n",
       " 'comic',\n",
       " 'book',\n",
       " '@-@',\n",
       " 'like',\n",
       " 'panels',\n",
       " 'with',\n",
       " 'animated',\n",
       " 'character',\n",
       " 'portraits',\n",
       " ',',\n",
       " 'with',\n",
       " 'characters',\n",
       " 'speaking',\n",
       " 'partially',\n",
       " 'through',\n",
       " 'voiced',\n",
       " 'speech',\n",
       " 'bubbles',\n",
       " 'and',\n",
       " 'partially',\n",
       " 'through',\n",
       " '<unk>',\n",
       " 'text',\n",
       " '.',\n",
       " 'the',\n",
       " 'player',\n",
       " 'progresses',\n",
       " 'through',\n",
       " 'a',\n",
       " 'series',\n",
       " 'of',\n",
       " 'linear',\n",
       " 'missions',\n",
       " ',',\n",
       " 'gradually',\n",
       " 'unlocked',\n",
       " 'as',\n",
       " 'maps',\n",
       " 'that',\n",
       " 'can',\n",
       " 'be',\n",
       " 'freely',\n",
       " '<unk>',\n",
       " 'through',\n",
       " 'and',\n",
       " 'replayed',\n",
       " 'as',\n",
       " 'they',\n",
       " 'are',\n",
       " 'unlocked',\n",
       " '.',\n",
       " 'the',\n",
       " 'route',\n",
       " 'to',\n",
       " 'each',\n",
       " 'story',\n",
       " 'location',\n",
       " 'on',\n",
       " 'the',\n",
       " 'map',\n",
       " 'varies',\n",
       " 'depending',\n",
       " 'on',\n",
       " 'an',\n",
       " 'individual',\n",
       " 'player',\n",
       " \"'\",\n",
       " 's',\n",
       " 'approach',\n",
       " 'when',\n",
       " 'one',\n",
       " 'option',\n",
       " 'is',\n",
       " 'selected',\n",
       " ',',\n",
       " 'the',\n",
       " 'other',\n",
       " 'is',\n",
       " 'sealed',\n",
       " 'off',\n",
       " 'to',\n",
       " 'the',\n",
       " 'player',\n",
       " '.',\n",
       " 'outside',\n",
       " 'missions',\n",
       " ',',\n",
       " 'the',\n",
       " 'player',\n",
       " 'characters',\n",
       " 'rest',\n",
       " 'in',\n",
       " 'a',\n",
       " 'camp',\n",
       " ',',\n",
       " 'where',\n",
       " 'units',\n",
       " 'can',\n",
       " 'be',\n",
       " 'customized',\n",
       " 'and',\n",
       " 'character',\n",
       " 'growth',\n",
       " 'occurs',\n",
       " '.',\n",
       " 'alongside',\n",
       " 'the',\n",
       " 'main',\n",
       " 'story',\n",
       " 'missions',\n",
       " 'are',\n",
       " 'character',\n",
       " '@-@',\n",
       " 'specific',\n",
       " 'sub',\n",
       " 'missions',\n",
       " 'relating',\n",
       " 'to',\n",
       " 'different',\n",
       " 'squad',\n",
       " 'members',\n",
       " '.',\n",
       " 'after',\n",
       " 'the',\n",
       " 'game',\n",
       " \"'\",\n",
       " 's',\n",
       " 'completion',\n",
       " ',',\n",
       " 'additional',\n",
       " 'episodes',\n",
       " 'are',\n",
       " 'unlocked',\n",
       " ',',\n",
       " 'some',\n",
       " 'of',\n",
       " 'them',\n",
       " 'having',\n",
       " 'a',\n",
       " 'higher',\n",
       " 'difficulty',\n",
       " 'than',\n",
       " 'those',\n",
       " 'found',\n",
       " 'in',\n",
       " 'the',\n",
       " 'rest',\n",
       " 'of',\n",
       " 'the',\n",
       " 'game',\n",
       " '.',\n",
       " 'there',\n",
       " 'are',\n",
       " 'also',\n",
       " 'love',\n",
       " 'simulation',\n",
       " 'elements',\n",
       " 'related',\n",
       " 'to',\n",
       " 'the',\n",
       " 'game',\n",
       " \"'\",\n",
       " 's',\n",
       " 'two',\n",
       " 'main',\n",
       " '<unk>',\n",
       " ',',\n",
       " 'although',\n",
       " 'they',\n",
       " 'take',\n",
       " 'a',\n",
       " 'very',\n",
       " 'minor',\n",
       " 'role',\n",
       " '.',\n",
       " '<eos>',\n",
       " 'the',\n",
       " 'game',\n",
       " \"'\",\n",
       " 's',\n",
       " 'battle',\n",
       " 'system',\n",
       " ',',\n",
       " 'the',\n",
       " '<unk>',\n",
       " 'system',\n",
       " ',',\n",
       " 'is',\n",
       " 'carried',\n",
       " 'over',\n",
       " 'directly',\n",
       " 'from',\n",
       " '<unk>',\n",
       " 'chronicles',\n",
       " '.',\n",
       " 'during',\n",
       " 'missions',\n",
       " ',',\n",
       " 'players',\n",
       " 'select',\n",
       " 'each',\n",
       " 'unit',\n",
       " 'using',\n",
       " 'a',\n",
       " 'top',\n",
       " '@-@',\n",
       " 'down',\n",
       " 'perspective',\n",
       " 'of',\n",
       " 'the',\n",
       " 'battlefield',\n",
       " 'map',\n",
       " 'once',\n",
       " 'a',\n",
       " 'character',\n",
       " 'is',\n",
       " 'selected',\n",
       " ',',\n",
       " 'the',\n",
       " 'player',\n",
       " 'moves',\n",
       " 'the',\n",
       " 'character',\n",
       " 'around',\n",
       " 'the',\n",
       " 'battlefield',\n",
       " 'in',\n",
       " 'third',\n",
       " '@-@',\n",
       " 'person',\n",
       " '.',\n",
       " 'a',\n",
       " 'character',\n",
       " 'can',\n",
       " 'only',\n",
       " 'act',\n",
       " 'once',\n",
       " 'per',\n",
       " '@-@',\n",
       " 'turn',\n",
       " ',',\n",
       " 'but',\n",
       " 'characters',\n",
       " 'can',\n",
       " 'be',\n",
       " 'granted',\n",
       " 'multiple',\n",
       " 'turns',\n",
       " 'at',\n",
       " 'the',\n",
       " 'expense',\n",
       " 'of',\n",
       " 'other',\n",
       " 'characters',\n",
       " \"'\",\n",
       " 'turns',\n",
       " '.',\n",
       " 'each',\n",
       " 'character',\n",
       " 'has',\n",
       " 'a',\n",
       " 'field',\n",
       " 'and',\n",
       " 'distance',\n",
       " 'of',\n",
       " 'movement',\n",
       " 'limited',\n",
       " 'by',\n",
       " 'their',\n",
       " 'action',\n",
       " '<unk>',\n",
       " '.',\n",
       " 'up',\n",
       " 'to',\n",
       " 'nine',\n",
       " 'characters',\n",
       " 'can',\n",
       " 'be',\n",
       " 'assigned',\n",
       " 'to',\n",
       " 'a',\n",
       " 'single',\n",
       " 'mission',\n",
       " '.',\n",
       " 'during',\n",
       " 'gameplay',\n",
       " ',',\n",
       " 'characters',\n",
       " 'will',\n",
       " 'call',\n",
       " 'out',\n",
       " 'if',\n",
       " 'something',\n",
       " 'happens',\n",
       " 'to',\n",
       " 'them',\n",
       " ',',\n",
       " 'such',\n",
       " 'as',\n",
       " 'their',\n",
       " 'health',\n",
       " 'points',\n",
       " '(',\n",
       " 'hp',\n",
       " ')',\n",
       " 'getting',\n",
       " 'low',\n",
       " 'or',\n",
       " 'being',\n",
       " 'knocked',\n",
       " 'out',\n",
       " 'by',\n",
       " 'enemy',\n",
       " 'attacks',\n",
       " '.',\n",
       " 'each',\n",
       " 'character',\n",
       " 'has',\n",
       " 'specific',\n",
       " 'potentials',\n",
       " ',',\n",
       " 'skills',\n",
       " 'unique',\n",
       " 'to',\n",
       " 'each',\n",
       " 'character',\n",
       " '.',\n",
       " 'they',\n",
       " 'are',\n",
       " 'divided',\n",
       " 'into',\n",
       " 'personal',\n",
       " 'potential',\n",
       " ',',\n",
       " 'which',\n",
       " 'are',\n",
       " 'innate',\n",
       " 'skills',\n",
       " 'that',\n",
       " 'remain',\n",
       " 'unaltered',\n",
       " 'unless',\n",
       " 'otherwise',\n",
       " 'dictated',\n",
       " 'by',\n",
       " 'the',\n",
       " 'story',\n",
       " 'and',\n",
       " 'can',\n",
       " 'either',\n",
       " 'help',\n",
       " 'or',\n",
       " 'impede',\n",
       " 'a',\n",
       " 'character',\n",
       " ',',\n",
       " 'and',\n",
       " 'battle',\n",
       " 'potentials',\n",
       " ',',\n",
       " 'which',\n",
       " 'are',\n",
       " 'grown',\n",
       " 'throughout',\n",
       " 'the',\n",
       " 'game',\n",
       " 'and',\n",
       " 'always',\n",
       " 'grant',\n",
       " '<unk>',\n",
       " 'to',\n",
       " 'a',\n",
       " 'character',\n",
       " '.',\n",
       " 'to',\n",
       " 'learn',\n",
       " 'battle',\n",
       " 'potentials',\n",
       " ',',\n",
       " 'each',\n",
       " 'character',\n",
       " 'has',\n",
       " 'a',\n",
       " 'unique',\n",
       " 'masters',\n",
       " 'table',\n",
       " ',',\n",
       " 'a',\n",
       " 'grid',\n",
       " '@-@',\n",
       " 'based',\n",
       " 'skill',\n",
       " 'table',\n",
       " 'that',\n",
       " 'can',\n",
       " 'be',\n",
       " 'used',\n",
       " 'to',\n",
       " 'acquire',\n",
       " 'and',\n",
       " 'link',\n",
       " 'different',\n",
       " 'skills',\n",
       " '.',\n",
       " 'characters',\n",
       " 'also',\n",
       " 'have',\n",
       " 'special',\n",
       " '<unk>',\n",
       " 'that',\n",
       " 'grant',\n",
       " 'them',\n",
       " 'temporary',\n",
       " '<unk>',\n",
       " 'on',\n",
       " 'the',\n",
       " 'battlefield',\n",
       " 'kurt',\n",
       " 'can',\n",
       " 'activate',\n",
       " 'direct',\n",
       " 'command',\n",
       " 'and',\n",
       " 'move',\n",
       " 'around',\n",
       " 'the',\n",
       " 'battlefield',\n",
       " 'without',\n",
       " '<unk>',\n",
       " 'his',\n",
       " 'action',\n",
       " 'point',\n",
       " 'gauge',\n",
       " ',',\n",
       " 'the',\n",
       " 'character',\n",
       " '<unk>',\n",
       " 'can',\n",
       " 'shift',\n",
       " 'into',\n",
       " 'her',\n",
       " 'valkyria',\n",
       " 'form',\n",
       " 'and',\n",
       " 'become',\n",
       " '<unk>',\n",
       " ',',\n",
       " 'while',\n",
       " 'imca',\n",
       " 'can',\n",
       " 'target',\n",
       " 'multiple',\n",
       " 'enemy',\n",
       " 'units',\n",
       " 'with',\n",
       " 'her',\n",
       " 'heavy',\n",
       " 'weapon',\n",
       " '.',\n",
       " '<eos>',\n",
       " 'troops',\n",
       " 'are',\n",
       " 'divided',\n",
       " 'into',\n",
       " 'five',\n",
       " 'classes',\n",
       " 'scouts',\n",
       " ',',\n",
       " '<unk>',\n",
       " ',',\n",
       " 'engineers',\n",
       " ',',\n",
       " '<unk>',\n",
       " 'and',\n",
       " 'armored',\n",
       " 'soldier',\n",
       " '.',\n",
       " '<unk>',\n",
       " 'can',\n",
       " 'switch',\n",
       " 'classes',\n",
       " 'by',\n",
       " 'changing',\n",
       " 'their',\n",
       " 'assigned',\n",
       " 'weapon',\n",
       " '.',\n",
       " 'changing',\n",
       " 'class',\n",
       " 'does',\n",
       " 'not',\n",
       " 'greatly',\n",
       " 'affect',\n",
       " 'the',\n",
       " 'stats',\n",
       " 'gained',\n",
       " 'while',\n",
       " 'in',\n",
       " 'a',\n",
       " 'previous',\n",
       " 'class',\n",
       " '.',\n",
       " 'with',\n",
       " 'victory',\n",
       " 'in',\n",
       " 'battle',\n",
       " ',',\n",
       " 'experience',\n",
       " 'points',\n",
       " 'are',\n",
       " 'awarded',\n",
       " 'to',\n",
       " 'the',\n",
       " 'squad',\n",
       " ',',\n",
       " 'which',\n",
       " 'are',\n",
       " 'distributed',\n",
       " 'into',\n",
       " 'five',\n",
       " 'different',\n",
       " 'attributes',\n",
       " 'shared',\n",
       " 'by',\n",
       " 'the',\n",
       " 'entire',\n",
       " 'squad',\n",
       " ',',\n",
       " 'a',\n",
       " 'feature',\n",
       " 'differing',\n",
       " 'from',\n",
       " 'early',\n",
       " 'games',\n",
       " \"'\",\n",
       " 'method',\n",
       " 'of',\n",
       " 'distributing',\n",
       " 'to',\n",
       " 'different',\n",
       " 'unit',\n",
       " 'types',\n",
       " '.',\n",
       " '<eos>',\n",
       " '<eos>',\n",
       " '=',\n",
       " '=',\n",
       " 'plot',\n",
       " '=',\n",
       " '=',\n",
       " '<eos>',\n",
       " '<eos>',\n",
       " 'the',\n",
       " 'game',\n",
       " 'takes',\n",
       " 'place',\n",
       " 'during',\n",
       " 'the',\n",
       " 'second',\n",
       " 'europan',\n",
       " 'war',\n",
       " '.',\n",
       " 'gallian',\n",
       " 'army',\n",
       " 'squad',\n",
       " '422',\n",
       " ',',\n",
       " 'also',\n",
       " 'known',\n",
       " 'as',\n",
       " 'the',\n",
       " 'nameless',\n",
       " ',',\n",
       " 'are',\n",
       " 'a',\n",
       " 'penal',\n",
       " 'military',\n",
       " 'unit',\n",
       " 'composed',\n",
       " 'of',\n",
       " 'criminals',\n",
       " ',',\n",
       " 'foreign',\n",
       " '<unk>',\n",
       " ',',\n",
       " 'and',\n",
       " 'military',\n",
       " 'offenders',\n",
       " 'whose',\n",
       " 'real',\n",
       " 'names',\n",
       " 'are',\n",
       " 'erased',\n",
       " 'from',\n",
       " 'the',\n",
       " 'records',\n",
       " 'and',\n",
       " '<unk>',\n",
       " 'officially',\n",
       " 'referred',\n",
       " 'to',\n",
       " 'by',\n",
       " 'numbers',\n",
       " '.',\n",
       " '<unk>',\n",
       " 'by',\n",
       " 'the',\n",
       " 'gallian',\n",
       " 'military',\n",
       " 'to',\n",
       " 'perform',\n",
       " 'the',\n",
       " 'most',\n",
       " 'dangerous',\n",
       " 'missions',\n",
       " 'that',\n",
       " 'the',\n",
       " 'regular',\n",
       " 'army',\n",
       " 'and',\n",
       " 'militia',\n",
       " 'will',\n",
       " 'not',\n",
       " 'do',\n",
       " ...]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T14:41:32.690719Z",
     "start_time": "2020-04-03T14:41:32.492713Z"
    }
   },
   "outputs": [],
   "source": [
    "ntokens = len(TEXT.vocab.stoi)\n",
    "emsize = 200\n",
    "nhid = 200\n",
    "nlayers = 2\n",
    "nhead = 2\n",
    "dropout = 0.2\n",
    "model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers,\n",
    "                         dropout).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T14:43:38.145372Z",
     "start_time": "2020-04-03T14:43:38.135820Z"
    }
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "lr = 5.0\n",
    "\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n",
    "\n",
    "import time\n",
    "\n",
    "\n",
    "def train():\n",
    "    model.train()\n",
    "    total_loss = 0.\n",
    "    start_time = time.time()\n",
    "    ntokens = len(TEXT.vocab.stoi)\n",
    "    for batch, i in enumerate(range(0, train_data.size(0) - 1, bptt)):\n",
    "        data, targets = get_batch(train_data, i)\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        output = model(data)\n",
    "        loss = criterion(output.view(-1, ntokens), targets)\n",
    "\n",
    "        loss.backward()\n",
    "\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n",
    "        optimizer.step()\n",
    "\n",
    "        total_loss += loss.item()\n",
    "        log_interval = 200\n",
    "        if batch % log_interval == 0 and batch > 0:\n",
    "            cur_loss = total_loss / log_interval\n",
    "            elapsed = time.time() - start_time\n",
    "            print(\n",
    "                \"|epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | loss {:5.2f} | ppl {:8.2f}\"\n",
    "                .format(epoch, batch,\n",
    "                        len(train_data) // bptt,\n",
    "                        scheduler.get_lr()[0], elapsed * 1000 / log_interval,\n",
    "                        cur_loss, math.exp(cur_loss)))\n",
    "        total_loss = 0\n",
    "        start_time = time.time()\n",
    "\n",
    "\n",
    "def evaluate(eval_model, data_source):\n",
    "    eval_model.eval()\n",
    "    total_loss = 0.\n",
    "    ntokens = len(TEXT.vocab.stoi)\n",
    "    with torch.no_grad():\n",
    "        for i in range(0, data_source.size(0) - 1, bptt):\n",
    "            data, targets = get_batch(data_source, i)\n",
    "            output = eval_model(data)\n",
    "            output_flat = output.view(-1, ntokens)\n",
    "            total_loss += len(data) * criterion(output_flat, targets).item()\n",
    "    return total_loss / (len(data_source) - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T14:45:00.346002Z",
     "start_time": "2020-04-03T14:43:38.954881Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|epoch   1 |   200/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |   400/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |   600/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |   800/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  1000/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  1200/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  1400/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  1600/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  1800/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  2000/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  2200/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  2400/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  2600/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   1 |  2800/ 2981 batches | lr 5.00 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "-----------------------------------------------------------------------------------------\n",
      "| end of epoch   1 | time: 26.96s | valid loss  5.60 | valid ppl   270.21\n",
      "-----------------------------------------------------------------------------------------\n",
      "|epoch   2 |   200/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |   400/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |   600/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |   800/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  1000/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  1200/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  1400/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  1600/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  1800/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  2000/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  2200/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  2400/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  2600/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   2 |  2800/ 2981 batches | lr 4.51 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "-----------------------------------------------------------------------------------------\n",
      "| end of epoch   2 | time: 27.13s | valid loss  5.57 | valid ppl   261.30\n",
      "-----------------------------------------------------------------------------------------\n",
      "|epoch   3 |   200/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |   400/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |   600/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |   800/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |  1000/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |  1200/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |  1400/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |  1600/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |  1800/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |  2000/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |  2200/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.02 | ppl     1.03\n",
      "|epoch   3 |  2400/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |  2600/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "|epoch   3 |  2800/ 2981 batches | lr 4.29 | ms/batch  0.04 | loss  0.03 | ppl     1.03\n",
      "-----------------------------------------------------------------------------------------\n",
      "| end of epoch   3 | time: 27.29s | valid loss  5.51 | valid ppl   248.01\n",
      "-----------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "best_val_loss = float(\"inf\")\n",
    "epochs = 3  # The number of epochs\n",
    "best_model = None\n",
    "\n",
    "for epoch in range(1, epochs + 1):\n",
    "    epoch_start_time = time.time()\n",
    "    train()\n",
    "    val_loss = evaluate(model, val_data)\n",
    "    print('-' * 89)\n",
    "    print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | '\n",
    "          'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time),\n",
    "                                     val_loss, math.exp(val_loss)))\n",
    "    print('-' * 89)\n",
    "\n",
    "    if val_loss < best_val_loss:\n",
    "        best_val_loss = val_loss\n",
    "        best_model = model\n",
    "\n",
    "    scheduler.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T14:46:00.659897Z",
     "start_time": "2020-04-03T14:45:59.140990Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========================================================================================\n",
      "| End of training | test loss  5.41 | test ppl   224.23\n",
      "=========================================================================================\n"
     ]
    }
   ],
   "source": [
    "test_loss = evaluate(best_model, test_data)\n",
    "print('=' * 89)\n",
    "print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(\n",
    "    test_loss, math.exp(test_loss)))\n",
    "print('=' * 89)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
